import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import glob
import os
import pickle
import re
import math
from utils import *
test_img_paths = glob.glob('test_images/test*.jpg')
video1 = glob.glob('video_frames/frame*.jpg')
video2 = glob.glob('video_frames_1/frame*.jpg')
# List of all demos to visualise
plot_demo = [1, 2, 3, 4, 5, 6, 7, 8]
def calibrate_camera():
imgpaths = glob.glob('camera_cal/cal_*.png')
image = cv2.imread(imgpaths[0])
imshape = image.shape[:2] # gets only the (height, width) to be used in the cv2.calibrateCamera()
objpoints = []
imgpoints = []
nx = 10 # Number of inside corners on each row of the chessboard
ny = 5 # Number of inside corners on each column of the chessboard
# Prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros([ny*nx, 3], dtype=np.float32)
objp[:,:2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)
img = cv2.imread(imgpaths[0])
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)
if ret:
img = cv2.drawChessboardCorners(img, (nx, ny), corners, ret)
imgpoints.append(corners)
objpoints.append(objp)
cv2.imshow('img', img)
cv2.waitKey(500)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, imshape[::-1], None, None)
cv2.destroyAllWindows()
return mtx, dist
if os.path.exists('camera_cal.p'):
with open('camera_cal.p', mode='rb') as f:
data = pickle.load(f)
mtx, dist = data['mtx'], data['dist']
print('Loaded the saved camera calibration matrix & dist coefficients!')
else:
mtx, dist = calibrate_camera()
with open('camera_cal.p', mode='wb') as f:
pickle.dump({'mtx': mtx, 'dist': dist}, f)
Loaded the saved camera calibration matrix & dist coefficients!
def undistort(img, mtx, dist):
undistort = cv2.undistort(img, mtx, dist, None, mtx)
return undistort
if 1 in plot_demo:
cal_img = cv2.imread('camera_cal/cal_01.png')
cal_img_undist = undistort(cal_img, mtx, dist)
plot_imgs([
(cal_img, 'Original Image'),
(cal_img_undist, 'Undistorted Image')
])
img_orig = mpimg.imread(test_img_paths[8])
img = undistort(img_orig, mtx, dist)
plot_imgs([
(img_orig, 'Original Image'),
(img, 'Undistorted Image')
])
IMG_SHAPE = (720, 1280)
def get_roi(img, vertices):
vertices = np.array(vertices, ndmin=3, dtype=np.int32)
if len(img.shape) == 3:
fill_color = (255,) * 3
else:
fill_color = 255
mask = np.zeros_like(img)
mask = cv2.fillPoly(mask, vertices, fill_color)
return cv2.bitwise_and(img, mask)
def warp_image(img, warp_shape, src, dst):
# Get the perspective transformation matrix and its inverse
M = cv2.getPerspectiveTransform(src, dst)
invM = cv2.getPerspectiveTransform(dst, src)
# Warp the image
warped = cv2.warpPerspective(img, M, warp_shape, flags=cv2.INTER_LINEAR)
return warped, M, invM
def preprocess_image(img, visualise=False):
ysize = img.shape[0]
xsize = img.shape[1]
x_offset = 0.015
x1_ratio = 0.18
x2_ratio = 0.46
y1_ratio = 0.63
y2_ratio = 0.95
# 1. Distortion correction
undist = undistort(img, mtx, dist)
# 2. Perspective transformation
src = np.float32([
(((1-x2_ratio)) * xsize, y1_ratio * ysize),
((x2_ratio) * xsize, y1_ratio * ysize),
((x1_ratio + x_offset) * xsize, y2_ratio * ysize),
(((1-x1_ratio) + x_offset) * xsize, y2_ratio * ysize)
])
dst = np.float32([
(xsize * (1 - ((x1_ratio + x2_ratio) / 2)), 0),
(xsize * ((x1_ratio + x2_ratio) / 2), 0),
(xsize * ((x1_ratio + x2_ratio) / 2), ysize),
(xsize * (1 - ((x1_ratio + x2_ratio) / 2)), ysize)
])
warped, M, invM = warp_image(undist, (xsize, ysize), src, dst)
# 3. ROI crop
vertices = np.array([
[200, ysize],
[200, 0],
[1100, 0],
[1100, ysize]
])
roi = get_roi(warped, vertices)
# 4. Visualise the transformation
if visualise:
img_copy = np.copy(img)
roi_copy = np.copy(roi)
cv2.polylines(img_copy, [np.int32(src)], True, (0, 255, 200), 3)
cv2.polylines(roi_copy, [np.int32(dst)], True, (0, 255, 200), 3)
plot_imgs([
(img_copy, 'Original Image'),
(roi_copy, 'Bird\'s Eye View Perspective')
])
return roi, (M, invM)
def get_image(img_path, visualise=False):
img = mpimg.imread(img_path)
return preprocess_image(img, visualise=visualise)
if 2 in plot_demo:
for path in test_img_paths[:]:
get_image(path, visualise=True)
def binary_threshold(img, low, high):
if len(img.shape) == 2:
output = np.zeros_like(img)
mask = (img >= low) & (img <= high)
elif len(img.shape) == 3:
output = np.zeros_like(img[:,:,0])
mask = (img[:,:,0] >= low[0]) & (img[:,:,0] <= high[0]) \
& (img[:,:,1] >= low[1]) & (img[:,:,1] <= high[1]) \
& (img[:,:,2] >= low[2]) & (img[:,:,2] <= high[2])
output[mask] = 1
return output
def get_binary_image(img, visualise=False):
### LAB color space
lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)
L = lab[:,:,0]
L_max, L_mean = np.max(L), np.mean(L)
B = lab[:,:,2]
B_max, B_mean = np.max(B), np.mean(B)
# YELLOW
L_adapt_yellow = max(80, int(L_max * 0.45))
B_adapt_yellow = max(int(B_max * 0.70), int(B_mean * 1.2))
lab_low_yellow = np.array((L_adapt_yellow, 120, B_adapt_yellow))
lab_high_yellow = np.array((255, 145, 255))
lab_yellow = binary_threshold(lab, lab_low_yellow, lab_high_yellow)
lab_binary = lab_yellow
### HSV color space
hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
H = hsv[:,:,0]
H_max, H_mean = np.max(H), np.mean(H)
S = hsv[:,:,1]
S_max, S_mean = np.max(S), np.mean(S)
V = hsv[:,:,2]
V_max, V_mean = np.max(V), np.mean(V)
# YELLOW
S_adapt_yellow = max(int(S_max * 0.25), int(S_mean * 1.75))
V_adapt_yellow = max(70, int(V_mean * 1.25))
hsv_low_yellow = np.array((15, S_adapt_yellow, V_adapt_yellow))
hsv_high_yellow = np.array((30, 255, 255))
hsv_yellow = binary_threshold(hsv, hsv_low_yellow, hsv_high_yellow)
# WHITE
V_adapt_white = max(150, int(V_max * 0.8),int(V_mean * 1.25))
hsv_low_white = np.array((0, 0, V_adapt_white))
hsv_high_white = np.array((255, 40, 255))
hsv_white = binary_threshold(hsv, hsv_low_white, hsv_high_white)
hsv_binary = hsv_yellow | hsv_white
### HLS color space
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
L = hls[:,:,1]
L_max, L_mean = np.max(L), np.mean(L)
S = hls[:,:,2]
S_max, S_mean = np.max(S), np.mean(S)
# YELLOW
L_adapt_yellow = max(80, int(L_mean * 1.25))
S_adapt_yellow = max(int(S_max * 0.25), int(S_mean * 1.75))
hls_low_yellow = np.array((15, L_adapt_yellow, S_adapt_yellow))
hls_high_yellow = np.array((30, 255, 255))
hls_yellow = binary_threshold(hls, hls_low_yellow, hls_high_yellow)
# WHITE
L_adapt_white = max(160, int(L_max *0.8),int(L_mean * 1.25))
hls_low_white = np.array((0, L_adapt_white, 0))
hls_high_white = np.array((255, 255, 255))
hls_white = binary_threshold(hls, hls_low_white, hls_high_white)
hls_binary = hls_yellow | hls_white
### R color channel (WHITE)
R = img[:,:,0]
# R = cv2.cvtColor(img, cv2.COLOR_RGB2YUV)[:,:,0]
R_max, R_mean = np.max(R), np.mean(R)
R_low_white = min(max(150, int(R_max * 0.55), int(R_mean * 1.95)),230)
R_binary = binary_threshold(R, R_low_white, 255)
B = img[:,:,2]
B_max, B_mean = np.max(B), np.mean(B)
B_low_white = min(max(160, int(B_max * 0.5), int(B_mean * 1.95)),210)
B_binary = binary_threshold(B, B_low_white, 255)
### Adaptive thresholding: Gaussian kernel
# YELLOW
adapt_yellow_S = cv2.adaptiveThreshold(hls[:,:,2], 1, \
cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 161, -5)
adapt_yellow_B = cv2.adaptiveThreshold(lab[:,:,2], 1, \
cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 161, -5)
adapt_yellow = adapt_yellow_S & adapt_yellow_B
# WHITE
adapt_white_R = cv2.adaptiveThreshold(img[:,:,0], 1, \
cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 161, -27)
adapt_white_V = cv2.adaptiveThreshold(hsv[:,:,2], 1, \
cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 161, -27)
adapt_white = adapt_white_R & adapt_white_V
adapt_binary = adapt_yellow | adapt_white
### Ensemble Voting
combined = np.asarray(lab_binary + hls_binary + hsv_binary + adapt_binary + B_binary, dtype=np.uint8)
combined[combined < 3] = 0
combined[combined >= 3] = 1
if visualise:
plot_imgs([
(img, 'Original'),
# (R_binary, 'R'),
(B_binary, 'B'),
(hls_binary, 'HLS'),
(hsv_binary, 'HSV'),
(lab_binary, 'LAB'),
(adapt_binary, 'Adaptive Thresh'),
(combined, 'Combined'),
# (hls_white, 'hls_white'),
# (hls_yellow, 'hls yellow'),
# (lab_yellow, 'lab yello'),
], figsize=(32, 42))
return combined
if 3 in plot_demo:
for img_path in test_img_paths[:9]: #video2[5:10]:
img, _ = get_image(img_path)
get_binary_image(img, visualise=True)
def get_poly_points(left_fit, right_fit):
ysize, xsize = IMG_SHAPE
# Get the points for the entire height of the image
plot_y = np.linspace(0, ysize-1, ysize)
plot_xleft = left_fit[0] * plot_y**2 + left_fit[1] * plot_y + left_fit[2]
plot_xright = right_fit[0] * plot_y**2 + right_fit[1] * plot_y + right_fit[2]
# But keep only those points that lie within the image
plot_xleft = plot_xleft[(plot_xleft >= 0) & (plot_xleft <= xsize - 1)]
plot_xright = plot_xright[(plot_xright >= 0) & (plot_xright <= xsize - 1)]
plot_yleft = np.linspace(ysize - len(plot_xleft), ysize - 1, len(plot_xleft))
plot_yright = np.linspace(ysize - len(plot_xright), ysize - 1, len(plot_xright))
return plot_xleft.astype(np.int), plot_yleft.astype(np.int), plot_xright.astype(np.int), plot_yright.astype(np.int)
def check_validity(left_fit, right_fit, diagnostics=False):
if left_fit is None or right_fit is None:
return False
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(left_fit, right_fit)
# Check whether the two lines lie within a plausible distance from one another for three distinct y-values
y1 = IMG_SHAPE[0] - 1 # Bottom
y2 = IMG_SHAPE[0] - int(min(len(plot_yleft), len(plot_yright)) * 0.35) # For the 2nd and 3rd, take values between y1 and the top-most available value.
y3 = IMG_SHAPE[0] - int(min(len(plot_yleft), len(plot_yright)) * 0.75)
# Check whether the line slopes are similar for two distinct y-values
# x = Ay**2 + By + C
# dx/dy = 2Ay + B
y1left_dx = 2 * left_fit[0] * y1 + left_fit[1]
y3left_dx = 2 * left_fit[0] * y3 + left_fit[1]
y1right_dx = 2 * right_fit[0] * y1 + right_fit[1]
y3right_dx = 2 * right_fit[0] * y3 + right_fit[1]
# Compute the L1-norm
norm1 = abs(y1left_dx - y1right_dx)
norm2 = abs(y3left_dx - y3right_dx)
# if diagnostics: print( norm1, norm2)
# Define the L1 norm threshold
thresh = 0.6 #0.45 #0.58
if (norm1 >= thresh) | (norm2 >= thresh):
if diagnostics:
print("Violated tangent criterion: " +
"norm1 == {:.3f}, norm2 == {:.3f} (thresh == {}).".format(norm1, norm2, thresh))
return False
return True
def polyfit_sliding_window(binary, lane_width_px=578, visualise=False, diagnostics=False):
global cache
ret = True
# Sanity check
if binary.max() <= 0:
return False, np.array([]), np.array([]), np.array([])
# Step 1: Compute the histogram along all the columns in the lower half of the image.
# The two most prominent peaks in this histogram will be good indicators of the
# x-position of the base of the lane lines
histogram = None
cutoffs = [int(binary.shape[0] / 2), 0]
for cutoff in cutoffs:
histogram = np.sum(binary[cutoff:, :], axis=0)
if histogram.max() > 0:
break
if histogram.max() == 0:
print('Unable to detect lane lines in this frame. Trying another frame!')
return False, np.array([]), np.array([])
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0] / 2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
if visualise:
plot_imgs([(binary, 'Binary')])
plt.plot(histogram, 'm', linewidth=4.0)
plt.plot((midpoint, midpoint), (0, IMG_SHAPE[0]), 'c')
plt.plot((0, IMG_SHAPE[1]), (cutoff, cutoff), 'c')
out = np.dstack((binary, binary, binary)) * 255
nb_windows = 12 # number of sliding windows
margin = 100 # width of the windows +/- margin
minpix = 50 # min number of pixels needed to recenter the window
window_height = int(IMG_SHAPE[0] / nb_windows)
min_lane_pts = 10 # min number of 'hot' pixels needed to fit a 2nd order polynomial as a
# lane line
# Identify the x-y positions of all nonzero pixels in the image
# Note: the indices here are equivalent to the coordinate locations of the
# pixel
nonzero = binary.nonzero()
nonzerox = np.array(nonzero[1])
nonzeroy = np.array(nonzero[0])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
for window in range(nb_windows):
# Identify window boundaries in x and y (and left and right)
win_y_low = IMG_SHAPE[0] - (1 + window) * window_height
win_y_high = IMG_SHAPE[0] - window * window_height
win_xleft_low = leftx_current - margin
win_xleft_high = leftx_current + margin
win_xright_low = rightx_current - margin
win_xright_high = rightx_current + margin
# Draw windows for visualisation
cv2.rectangle(out, (win_xleft_low, win_y_low), (win_xleft_high, win_y_high),\
(0, 255, 0), 2)
cv2.rectangle(out, (win_xright_low, win_y_low), (win_xright_high, win_y_high),\
(0, 255, 0), 2)
# Identify the nonzero pixels in x and y within the window
good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy <= win_y_high)
& (nonzerox >= win_xleft_low) & (nonzerox <= win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy <= win_y_high)
& (nonzerox >= win_xright_low) & (nonzerox <= win_xright_high)).nonzero()[0]
left_lane_inds.append(good_left_inds)
right_lane_inds.append(good_right_inds)
# If you found > minpix pixels, recenter next window on their mean position
if len(good_left_inds) > minpix:
leftx_current = int(np.mean(nonzerox[good_left_inds]))
if len(good_right_inds) > minpix:
rightx_current = int(np.mean(nonzerox[good_right_inds]))
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
# Extract pixel positions for the left and right lane lines
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
left_fit, right_fit = None, None
# Sanity check; Fit a 2nd order polynomial for each lane line pixels
if len(leftx) >= min_lane_pts and len(rightx) >= min_lane_pts:
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Validate detected lane lines
valid = check_validity(left_fit, right_fit, diagnostics=diagnostics)
if not valid:
# If the detected lane lines are NOT valid:
# 1. Compute the lane lines as an average of the previously detected lines
# from the cache and flag this detection cycle as a failure by setting ret=False
# 2. Else, if cache is empty, return
if len(cache) == 0:
if diagnostics: print('WARNING: Unable to detect lane lines in this frame.')
return False, np.array([]), np.array([])
avg_params = np.mean(cache, axis=0)
left_fit, right_fit = avg_params[0], avg_params[1]
ret = False
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(left_fit, right_fit)
# Color the detected pixels for each lane line
out[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
out[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [255, 10, 255]
left_poly_pts = np.array([np.transpose(np.vstack([plot_xleft, plot_yleft]))])
right_poly_pts = np.array([np.transpose(np.vstack([plot_xright, plot_yright]))])
# Plot the fitted polynomial
cv2.polylines(out, np.int32([left_poly_pts]), isClosed=False, color=(200,255,155), thickness=4)
cv2.polylines(out, np.int32([right_poly_pts]), isClosed=False, color=(200,255,155), thickness=4)
if visualise:
plot_imgs([(img, 'Original'), (out, 'Out')], figsize=(30, 40))
return ret, out, np.array([left_fit, right_fit])
if 4 in plot_demo:
cache = np.array([])
for img_path in test_img_paths[:]: #video2[134:138]:
img, _ = get_image(img_path)
binary = get_binary_image(img, visualise=False)
polyfit_sliding_window(binary, visualise=True, diagnostics=True)
C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app
Violated tangent criterion: norm1 == 1.203, norm2 == 0.244 (thresh == 0.6). WARNING: Unable to detect lane lines in this frame.
def polyfit_adapt_search(img, prev_poly_param, visualise=False, diagnostics=False):
global cache # Cache of the previosuly detected lane line coefficients
global attempts # Number of retries before the pipeline is RESET to detect lines via the smoothing window aproach
# Sanity check
assert(len(img.shape) == 3)
# Setup
nb_windows = 10 # Number of windows over which to perform the localised color thresholding
bin_margin = 80 # Width of the windows +/- margin for localised thresholding
margin = 60 # Width around previous line positions +/- margin around which to search for the new lines
window_height = int(img.shape[0] / nb_windows)
smoothing_window = 5 # Number of frames over which to compute the Moving Average
min_lane_pts = 10
binary = np.zeros_like(img[:,:,0]) # Placeholder for the thresholded binary image
img_plot = np.copy(img)
left_fit, right_fit = prev_poly_param[0], prev_poly_param[1]
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(left_fit, right_fit)
leftx_current = np.int(plot_xleft[-1])
rightx_current = np.int(plot_xright[-1])
# Iterate over the windows, perform localised color thresholding and generate the binary image
for window in range(nb_windows):
# Identify window boundaries in x and y (and left and right)
win_y_low = IMG_SHAPE[0] - (window + 1) * window_height
win_y_high = IMG_SHAPE[0] - window * window_height
win_xleft_low = min(max(0, leftx_current - bin_margin), 1280)
win_xleft_high = min(max(0, leftx_current + bin_margin), 1280)
win_xright_low = min(max(0, rightx_current - bin_margin), 1280)
win_xright_high = min(max(0, rightx_current + bin_margin), 1280)
img_win_left = img[win_y_low:win_y_high, win_xleft_low:win_xleft_high,:]
binary[win_y_low:win_y_high, win_xleft_low:win_xleft_high] = \
get_binary_image(img_win_left, visualise=False)
img_win_right = img[win_y_low:win_y_high, win_xright_low:win_xright_high, :]
binary[win_y_low:win_y_high, win_xright_low:win_xright_high] = \
get_binary_image(img_win_right, visualise=False)
# Given that we only keep the points/values for a line that lie within the image
# (see 'get_poly_points'), the overall length and consequently number of points (i.e. x-values
# and y-values) can be < the length of the image. As a result, we check for the presence
# of the current window's lower y-value i.e 'win_y_low' as a valid point within the previously detected line
# If, a point associated with this y-value exists, we update the x-position of the next window with
# the corresponding x-value.
# Else, we keep the x-position of the subsequent windows the same and move up the image
idxs = np.where(plot_yleft == win_y_low)[0]
if len(idxs) != 0:
leftx_current = int(plot_xleft[idxs[0]])
idxs = np.where(plot_yright == win_y_low)[0]
if len(idxs) != 0:
rightx_current = int(plot_xright[idxs[0]])
if visualise:
left_pts = np.array([np.transpose(np.vstack([plot_xleft, plot_yleft]))])
right_pts = np.array([np.transpose(np.vstack([plot_xright, plot_yright]))])
# Plot the previously detected lane lines
cv2.polylines(img_plot, np.int32([left_pts]), isClosed=False, color=(255, 20, 147), thickness=4)
cv2.polylines(img_plot, np.int32([right_pts]), isClosed=False, color=(255, 20, 147), thickness=4)
bin_win_left = binary[win_y_low:win_y_high, win_xleft_low:win_xleft_high]
bin_win_left = np.dstack((bin_win_left, np.zeros_like(bin_win_left), np.zeros_like(bin_win_left))) * 255
bin_win_right = binary[win_y_low:win_y_high, win_xright_low:win_xright_high]
bin_win_right = np.dstack([np.zeros_like(bin_win_right), np.zeros_like(bin_win_right), bin_win_right]) * 255
# Blend the localised image window with its corresponding thresholded binary version
win_left = cv2.addWeighted(bin_win_left, 0.5, img_win_left, 0.7, 0)
win_right = cv2.addWeighted(bin_win_right, 0.5, img_win_right, 0.7, 0)
# Draw the binary search window
cv2.rectangle(img_plot, (win_xleft_low,win_y_low), (win_xleft_high,win_y_high), (0,255,0), 5)
cv2.rectangle(img_plot, (win_xright_low,win_y_low), (win_xright_high,win_y_high), (0,255,0), 5)
f, _ = plt.subplots(1, 2, figsize=(13,5))
plt.subplot(121)
plt.axis('off')
plt.imshow(binary, cmap='gray')
plt.subplot(122)
plt.axis('off')
plt.imshow(img_plot)
plt.subplots_adjust(top=0.98, bottom=0.0, left=0.0, right=1.0, hspace=0.1, wspace=0.05)
plt.savefig('./gif_images/window1{:02}.png'.format(window))
# The blended Binary window and Image window is added later for better visualisation
img_plot[win_y_low:win_y_high, win_xleft_low:win_xleft_high] = win_left
img_plot[win_y_low:win_y_high, win_xright_low:win_xright_high] = win_right
# Identify the x-y coordinates of all the non-zero pixels from the binary image
# generated above
nonzero = binary.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Extract all the
left_lane_inds = \
((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] - margin)) &
(nonzerox < (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] + margin)))
right_lane_inds = \
((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] - margin)) &
(nonzerox < (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] + margin)))
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Sanity checks
if len(leftx) > min_lane_pts:
left_fit = np.polyfit(lefty, leftx, 2)
else:
if diagnostics: print('WARNING: Less than {} pts detected for the left lane. {}'.format(min_lane_pts, len(leftx)))
if len(rightx) > min_lane_pts:
right_fit = np.polyfit(righty, rightx, 2)
else:
if diagnostics: print('WARNING: Less than {} pts detected for the right lane. {}'.format(min_lane_pts, len(rightx)))
valid = check_validity(left_fit, right_fit, diagnostics=diagnostics)
# Perform smoothing via moving average
if valid:
if len(cache) < smoothing_window:
cache = np.concatenate((cache, [np.array([left_fit, right_fit])]), axis=0)
elif len(cache) >= smoothing_window:
cache[:-1] = cache[1:]
cache[-1] = np.array([left_fit, right_fit])
avg_params = np.mean(cache, axis=0)
left_fit, right_fit = avg_params[0], avg_params[1]
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(left_fit, right_fit)
curr_poly_param = np.array([left_fit, right_fit])
else:
attempts += 1
curr_poly_param = prev_poly_param
out = np.dstack([binary, binary, binary]) * 255
win_img = np.zeros_like(out)
# Color the lane line pixels
out[lefty, leftx] = [255, 0, 0]
out[righty, rightx] = [255, 10, 255]
left_window1 = np.array([np.transpose(np.vstack([plot_xleft - margin, plot_yleft]))])
left_window2 = np.array([np.flipud(np.transpose(np.vstack([plot_xleft + margin, plot_yleft])))])
left_pts = np.hstack([left_window1, left_window2])
right_window1 = np.array([np.transpose(np.vstack([plot_xright - margin, plot_yright]))])
right_window2 = np.array([np.flipud(np.transpose(np.vstack([plot_xright + margin, plot_yright])))])
right_pts = np.hstack([right_window1, right_window2])
# Draw the search boundary
cv2.fillPoly(win_img, np.int_([left_pts]), (0, 255, 0))
cv2.fillPoly(win_img, np.int_([right_pts]), (0, 255, 0))
out = cv2.addWeighted(out, 1, win_img, 0.25, 0)
left_poly_pts = np.array([np.transpose(np.vstack([plot_xleft, plot_yleft]))])
right_poly_pts = np.array([np.transpose(np.vstack([plot_xright, plot_yright]))])
# Draw the fit lane lines
cv2.polylines(out, np.int32([left_poly_pts]), isClosed=False, color=(200,255,155), thickness=4)
cv2.polylines(out, np.int32([right_poly_pts]), isClosed=False, color=(200,255,155), thickness=4)
return out, curr_poly_param
if 5 in plot_demo:
cache = np.array([])
attempts = 0
max_attempts = 4
reset = True
for frame_path in test_img_paths:
img = mpimg.imread(frame_path)
warped, (M, invM) = get_image(frame_path)
if reset == True:
binary = get_binary_image(warped)
ret, out, poly_param = polyfit_sliding_window(binary, visualise=False, diagnostics=True)
if ret:
reset = False
cache = np.array([poly_param])
else:
out, poly_param = polyfit_adapt_search(warped, poly_param, visualise=False, diagnostics=False)
if attempts == max_attempts:
attempts = 0
reset = True
out_unwarped = cv2.warpPerspective(out, invM, (IMG_SHAPE[1], IMG_SHAPE[0]), flags=cv2.INTER_LINEAR)
img_overlay = np.copy(img)
img_overlay = cv2.addWeighted(out_unwarped, 0.5, img, 0.5, 0)
plot_imgs([(warped, 'Original'), (out, 'Out'), (img_overlay, 'Overlay')], figsize=(20, 18))
C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:30: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:23: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:24: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
def compute_mppx(img, dashed_line_loc, visualise=False):
lane_width = 3.7
dashed_line_len = 3.048
if dashed_line_loc == 'left':
y_top = 330
y_bottom = 430
elif dashed_line_loc == 'right':
y_top = 427
y_bottom = 517
binary = get_binary_image(img)
histogram = np.sum(binary[int(binary.shape[0] / 2):, :], axis=0)
midpoint = np.int(histogram.shape[0] / 2)
x_left = np.argmax(histogram[:midpoint])
x_right = np.argmax(histogram[midpoint:]) + midpoint
x_mppx = lane_width / (x_right - x_left)
y_mppx = dashed_line_len / (y_bottom - y_top)
if visualise:
plt.figure(figsize=(10, 8))
plt.imshow(img)
plt.axis('off')
if dashed_line_loc == 'left':
plt.plot((x_left, x_left), (y_top, y_bottom), 'r')
plt.text(x_left + 10, (y_top + y_bottom) // 2, '{} m'.format(dashed_line_len), color='r', fontsize=20)
elif dashed_line_loc == 'right':
plt.plot((x_right, x_right), (y_top, y_bottom), 'r')
plt.text(x_right + 10, (y_top + y_bottom) // 2, '{} m'.format(dashed_line_len), color='r',fontsize=20)
plt.plot((x_left, x_right), (img.shape[0] - 200 , img.shape[0] - 200), 'r')
plt.text((x_left + x_right) // 2, img.shape[0] - 220, '{} m'.format(lane_width), color='r', fontsize=20)
return y_mppx, x_mppx
if 6 in plot_demo:
visualise = True
else:
visualise = False
img, _ = get_image(test_img_paths[0])
y_mppx1, x_mppx1 = compute_mppx(img, dashed_line_loc='right', visualise=visualise)
img, _ = get_image(test_img_paths[1])
y_mppx2, x_mppx2 = compute_mppx(img, dashed_line_loc='left', visualise=visualise)
x_mppx = (x_mppx1 + x_mppx2) / 2
y_mppx = (y_mppx1 + y_mppx2) / 2
print('Average meter/px along x-axis: {:.4f}'.format(x_mppx))
print('Average meter/px along y-axis: {:.4f}'.format(y_mppx))
C:\Users\MarkM\anaconda3\envs\pytorch_env\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations from ipykernel import kernelapp as app
Average meter/px along x-axis: 0.0083 Average meter/px along y-axis: 0.0322
def compute_offset_from_center(poly_param, x_mppx):
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(poly_param[0], poly_param[1])
lane_center = (plot_xright[-1] + plot_xleft[-1]) / 2
car_center = IMG_SHAPE[1] / 2
offset = (lane_center - car_center) * x_mppx
return offset
def compute_curvature(poly_param, y_mppx, x_mppx):
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(poly_param[0], poly_param[1])
y_eval = np.max(plot_yleft)
left_fit_cr = np.polyfit(plot_yleft * y_mppx, plot_xleft * x_mppx, 2)
right_fit_cr = np.polyfit(plot_yright * y_mppx, plot_xright * x_mppx, 2)
left_curverad = ((1 + (2*left_fit_cr[0]* y_eval*y_mppx + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*y_mppx + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
return left_curverad, right_curverad
def draw(img, warped, invM, poly_param, curve_rad, offset):
undist = undistort(img, mtx, dist)
warp_zero = np.zeros_like(warped[:,:,0]).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
left_fit = poly_param[0]
right_fit = poly_param[1]
plot_xleft, plot_yleft, plot_xright, plot_yright = get_poly_points(left_fit, right_fit)
pts_left = np.array([np.transpose(np.vstack([plot_xleft, plot_yleft]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([plot_xright, plot_yright])))])
pts = np.hstack((pts_left, pts_right))
# Color the road
cv2.fillPoly(color_warp, np.int_([pts]), (0, 220, 110))
cv2.polylines(color_warp, np.int32([pts_left]), isClosed=False,
color=(255, 255, 255), thickness=10)
cv2.polylines(color_warp, np.int32([pts_right]), isClosed=False,
color=(255, 255, 255), thickness= 10)
# Unwarp and merge with undistorted original image
unwarped = cv2.warpPerspective(color_warp, invM, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR)
out = cv2.addWeighted(undist, 1, unwarped, 0.4, 0)
# Write data on the image
if (left_fit[1] + right_fit[1]) / 2 > 0.05:
text = 'Left turn, curve radius: {:04.2f} m'.format(curve_rad)
elif (left_fit[1] + right_fit[1]) / 2 < -0.05:
text = 'Right turn, curve radius: {:04.2f} m'.format(curve_rad)
else:
text = 'Straight'
cv2.putText(out, text, (50, 60), cv2.FONT_HERSHEY_DUPLEX, 1.2, (255, 255, 255), 2, cv2.LINE_AA)
direction = ''
if offset > 0:
direction = 'left'
elif offset < 0:
direction = 'right'
text = '{:0.1f} cm {} of center'.format(abs(offset) * 100, direction)
cv2.putText(out, text, (50, 110), cv2.FONT_HERSHEY_DUPLEX, 1.2, (255, 255, 255), 2, cv2.LINE_AA)
return out
if 7 in plot_demo:
cache = np.array([])
for img_path in video1[1000: 1010]:#test_img_paths:
img = mpimg.imread(img_path)
warped, (M, invM) = get_image(img_path)
binary = get_binary_image(warped)
ret, img_poly, poly_param = polyfit_sliding_window(binary)
left_curverad, right_curverad = compute_curvature(poly_param, y_mppx, x_mppx)
curvature = (left_curverad + right_curverad) / 2
offset = compute_offset_from_center(poly_param, x_mppx)
result = draw(img, warped, invM, poly_param, curvature, offset)
plot_imgs([(img_poly, 'Polyfit'), (result, 'Result')])
def pipeline(img, visualise=False, diagnostics=False, debug=False):
global cache
global poly_param # Important for successive calls to the pipeline
global attempts
global reset
max_attempts = 5
result = np.copy(img)
warped, (M, invM) = preprocess_image(img)
title = ''
try:
if reset == True:
title = 'Sliding window'
if diagnostics: print(title)
binary = get_binary_image(warped)
ret, img_poly, poly_param = polyfit_sliding_window(binary, diagnostics=diagnostics)
if ret:
if diagnostics: print('Success!')
reset = False
cache = np.array([poly_param])
else:
if len(img_poly) == 0:
print('Sliding window failed!')
return img
else:
title = 'Adaptive Search'
if diagnostics: print(title)
img_poly, poly_param = polyfit_adapt_search(warped, poly_param, diagnostics=diagnostics)
if attempts == max_attempts:
if diagnostics: print('Resetting...')
reset = True
attempts = 0
img1, _ = get_image(test_img_paths[0])
y_mppx1, x_mppx1 = compute_mppx(img1, dashed_line_loc='right', visualise=False)
img2, _ = get_image(test_img_paths[1])
y_mppx2, x_mppx2 = compute_mppx(img2, dashed_line_loc='left', visualise=False)
x_mppx = (x_mppx1 + x_mppx2) / 2
y_mppx = (y_mppx1 + y_mppx2) / 2
left_curverad, right_curverad = compute_curvature(poly_param, y_mppx, x_mppx)
offset = compute_offset_from_center(poly_param, x_mppx)
result = draw(img, warped, invM, poly_param, (left_curverad + right_curverad) / 2, offset)
blended_warped_poly = cv2.addWeighted(img_poly, 0.6, warped, 1, 0)
if debug:
binary = get_binary_image(warped)
warped_ = cv2.resize(warped, (640, 360),
interpolation = cv2.INTER_NEAREST)
binary_ = cv2.resize(binary, (640, 360),
interpolation = cv2.INTER_NEAREST)
binary_ = np.dstack([binary_, binary_, binary_]) * 255
img_poly_ = cv2.resize(img_poly, (640, 360),
interpolation = cv2.INTER_NEAREST)
blended_warped_poly_ = cv2.resize(blended_warped_poly, (640, 360),
interpolation = cv2.INTER_NEAREST)
ret2 = np.hstack([warped_, binary_, img_poly_, blended_warped_poly_])
ret3 = np.hstack([img, result])
ret3 = np.vstack([ret2, ret3])
if visualise:
plt.figure(figsize=(20, 12))
plt.title(title)
plt.imshow(ret3)
if debug:
return ret3
else:
return result
except Exception as e:
print(e)
return img
if 8 in plot_demo:
print('Demo of consecutive frames')
cache = np.array([])
attempts = 0
reset = True
for img_path in video1[0:1]:
img = mpimg.imread(img_path)
result = pipeline(img, visualise=True, diagnostics=1)
Demo of consecutive frames
from moviepy.editor import VideoFileClip
from IPython.display import HTML
process_frame = lambda frame: pipeline(frame, diagnostics=0)
# Pipeline initialisation
cache = np.array([])
attempts = 0
reset = True
# video_output = 'project_video_out.mp4'
# video_input = VideoFileClip('project_video.mp4')
# processed_video = video_input.fl_image(process_frame)
# %time processed_video.write_videofile(video_output, audio=False)
video1_output = 'challenge_video_out1.mp4'
video1_input = VideoFileClip('challenge_video.mp4')
processed_video = video1_input.fl_image(process_frame)
%time processed_video.write_videofile(video1_output, audio=False)
# video2_output = 'harder_challenge_video_out.mp4'
# video2_input = VideoFileClip('harder_challenge_video.mp4')
# processed_video = video2_input.fl_image(process_frame)
# %time processed_video.write_videofile(video2_output, audio=False)
t: 4%|███▊ | 19/485 [00:56<12:00, 1.55s/it, now=None]
Moviepy - Building video challenge_video_out1.mp4. Moviepy - Writing video challenge_video_out1.mp4
t: 0%| | 0/60 [00:00<?, ?it/s, now=None] t: 3%|███▎ | 2/60 [00:01<00:46, 1.24it/s, now=None] t: 5%|█████ | 3/60 [00:03<01:01, 1.07s/it, now=None] t: 7%|██████▋ | 4/60 [00:04<01:06, 1.18s/it, now=None] t: 8%|████████▎ | 5/60 [00:05<01:08, 1.25s/it, now=None] t: 10%|██████████ | 6/60 [00:07<01:11, 1.33s/it, now=None] t: 12%|███████████▋ | 7/60 [00:08<01:11, 1.35s/it, now=None] t: 13%|█████████████▎ | 8/60 [00:10<01:11, 1.38s/it, now=None] t: 15%|███████████████ | 9/60 [00:11<01:10, 1.38s/it, now=None] t: 17%|████████████████▌ | 10/60 [00:13<01:15, 1.52s/it, now=None] t: 18%|██████████████████▏ | 11/60 [00:15<01:22, 1.68s/it, now=None] t: 20%|███████████████████▊ | 12/60 [00:17<01:27, 1.82s/it, now=None] t: 22%|█████████████████████▍ | 13/60 [00:19<01:25, 1.82s/it, now=None] t: 23%|███████████████████████ | 14/60 [00:21<01:25, 1.85s/it, now=None] t: 25%|████████████████████████▊ | 15/60 [00:23<01:24, 1.88s/it, now=None] t: 27%|██████████████████████████▍ | 16/60 [00:24<01:17, 1.76s/it, now=None] t: 28%|████████████████████████████ | 17/60 [00:26<01:12, 1.69s/it, now=None] t: 30%|█████████████████████████████▋ | 18/60 [00:27<01:06, 1.60s/it, now=None] t: 32%|███████████████████████████████▎ | 19/60 [00:29<01:02, 1.54s/it, now=None] t: 33%|█████████████████████████████████ | 20/60 [00:30<00:59, 1.49s/it, now=None] t: 35%|██████████████████████████████████▋ | 21/60 [00:31<00:56, 1.45s/it, now=None] t: 37%|████████████████████████████████████▎ | 22/60 [00:33<00:54, 1.44s/it, now=None] t: 38%|█████████████████████████████████████▉ | 23/60 [00:34<00:54, 1.47s/it, now=None] t: 40%|███████████████████████████████████████▌ | 24/60 [00:36<00:53, 1.49s/it, now=None] t: 42%|█████████████████████████████████████████▎ | 25/60 [00:37<00:51, 1.46s/it, now=None] t: 43%|██████████████████████████████████████████▉ | 26/60 [00:38<00:48, 1.43s/it, now=None] t: 45%|████████████████████████████████████████████▌ | 27/60 [00:40<00:47, 1.43s/it, now=None] t: 47%|██████████████████████████████████████████████▏ | 28/60 [00:41<00:45, 1.42s/it, now=None] t: 48%|███████████████████████████████████████████████▊ | 29/60 [00:43<00:43, 1.40s/it, now=None] t: 50%|█████████████████████████████████████████████████▌ | 30/60 [00:44<00:41, 1.39s/it, now=None] t: 52%|███████████████████████████████████████████████████▏ | 31/60 [00:46<00:41, 1.42s/it, now=None] t: 53%|████████████████████████████████████████████████████▊ | 32/60 [00:47<00:40, 1.44s/it, now=None] t: 55%|██████████████████████████████████████████████████████▍ | 33/60 [00:49<00:39, 1.46s/it, now=None] t: 57%|████████████████████████████████████████████████████████ | 34/60 [00:50<00:38, 1.50s/it, now=None] t: 58%|█████████████████████████████████████████████████████████▊ | 35/60 [00:52<00:37, 1.51s/it, now=None] t: 60%|███████████████████████████████████████████████████████████▍ | 36/60 [00:53<00:35, 1.47s/it, now=None] t: 62%|█████████████████████████████████████████████████████████████ | 37/60 [00:54<00:33, 1.45s/it, now=None] t: 63%|██████████████████████████████████████████████████████████████▋ | 38/60 [00:56<00:31, 1.43s/it, now=None] t: 65%|████████████████████████████████████████████████████████████████▎ | 39/60 [00:57<00:29, 1.42s/it, now=None] t: 67%|██████████████████████████████████████████████████████████████████ | 40/60 [00:59<00:29, 1.46s/it, now=None] t: 68%|███████████████████████████████████████████████████████████████████▋ | 41/60 [01:00<00:27, 1.46s/it, now=None] t: 70%|█████████████████████████████████████████████████████████████████████▎ | 42/60 [01:02<00:27, 1.52s/it, now=None] t: 72%|██████████████████████████████████████████████████████████████████████▉ | 43/60 [01:03<00:25, 1.52s/it, now=None] t: 73%|████████████████████████████████████████████████████████████████████████▌ | 44/60 [01:05<00:24, 1.54s/it, now=None] t: 75%|██████████████████████████████████████████████████████████████████████████▎ | 45/60 [01:06<00:22, 1.50s/it, now=None] t: 77%|███████████████████████████████████████████████████████████████████████████▉ | 46/60 [01:08<00:20, 1.48s/it, now=None] t: 78%|█████████████████████████████████████████████████████████████████████████████▌ | 47/60 [01:09<00:18, 1.45s/it, now=None] t: 80%|███████████████████████████████████████████████████████████████████████████████▏ | 48/60 [01:11<00:17, 1.44s/it, now=None] t: 82%|████████████████████████████████████████████████████████████████████████████████▊ | 49/60 [01:12<00:15, 1.44s/it, now=None] t: 83%|██████████████████████████████████████████████████████████████████████████████████▌ | 50/60 [01:14<00:14, 1.46s/it, now=None] t: 85%|████████████████████████████████████████████████████████████████████████████████████▏ | 51/60 [01:15<00:13, 1.48s/it, now=None] t: 87%|█████████████████████████████████████████████████████████████████████████████████████▊ | 52/60 [01:17<00:12, 1.52s/it, now=None] t: 88%|███████████████████████████████████████████████████████████████████████████████████████▍ | 53/60 [01:18<00:10, 1.50s/it, now=None] t: 90%|█████████████████████████████████████████████████████████████████████████████████████████ | 54/60 [01:20<00:08, 1.48s/it, now=None] t: 92%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 55/60 [01:21<00:07, 1.45s/it, now=None] t: 93%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 56/60 [01:22<00:05, 1.42s/it, now=None] t: 95%|██████████████████████████████████████████████████████████████████████████████████████████████ | 57/60 [01:24<00:04, 1.46s/it, now=None] t: 97%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 58/60 [01:25<00:02, 1.47s/it, now=None] t: 98%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 59/60 [01:27<00:01, 1.48s/it, now=None] t: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [01:28<00:00, 1.49s/it, now=None] t: 4%|███▊ | 19/485 [02:29<12:00, 1.55s/it, now=None]
Moviepy - Done ! Moviepy - video ready challenge_video_out1.mp4 Wall time: 1min 32s